package com.kin.rich.service;

import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Date;

import org.cyberneko.html.parsers.DOMParser;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.FileSystemXmlApplicationContext;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;

import com.kin.rich.pojo.Company;
import com.kin.rich.pojo.Keyword;
import com.kin.rich.pojo.Project;
import com.kin.rich.pojo.Searchdata;
import com.kin.rich.pojo.User;
import com.kin.rich.util.CalendarFunction;


public class BaiduNewsSpider extends AbstractSpider implements Runnable{
	
	private String string = "";
	
	private Keyword initKeyword;
	
	public int pageNum = 0;
	
	public int sameNews = 0;
	
	public boolean continueNews = false;
	
	public BaiduNewsSpider(){
		
	}
	
	public BaiduNewsSpider(Keyword keyword){
		this.initKeyword = keyword;
	}
	public void run(){
		this.Search(null,initKeyword);
	}
	
	public void search(Keyword keyword) {
		//log.error("start");
		String word = keyword.getWordName();
		int pageNum = keyword.getCountPageNo();
		log.error("页数："+pageNum);
		int wordId = keyword.getWordId();
		try{
			
			String uri = "http://news.baidu.com/ns?word=%"+URLEncoder.encode(word, "GBK")+"%C0&ie=gb2312&bs=%D0%C7%C3%C0&sr=0&cl=2&rn=20&tn=newsA&ct=0&clk=sortbytime";
			Date date = new Date();
			String today = CalendarFunction.dateToString(date);
			//检查今日是否有搜索
			//System.out.println("ssss"+helper==null);
			int searchTimes = helper.getSearchTimes(wordId, today, "BaiduNews");
			log.error("searchTimes   :"+searchTimes);
			searchTimes = searchTimes >=0 ? (searchTimes+1) : 1;
			log.error("searchTimes::::"+searchTimes);
			for(int i=1;i<=pageNum;i++){
				
					string = "";
					URL url = new URL(uri);
					log.error("url:"+url);
					HttpURLConnection connection = (HttpURLConnection)url.openConnection();
					connection.setConnectTimeout(5000);
					InputStreamReader in = new InputStreamReader(connection.getInputStream(),"GB18030");
					
					DOMParser parser = new DOMParser();
			        parser.setProperty("http://cyberneko.org/html/properties/default-encoding","GB18030");
			        parser.setFeature("http://cyberneko.org/html/features/augmentations", true);
			        parser.setProperty("http://cyberneko.org/html/properties/names/elems", "lower");
			        
			        parser.parse(new InputSource(in));
			        Document doc = parser.getDocument();
			        Node html = doc.getElementsByTagName("html").item(0);
			        Searchdata source = new Searchdata();
			        source.setWordId(wordId);
			        source.setDataSource(this.writeDoc(html));
			        source.setDataPageNo(i);
			        source.setLinkUrl(uri+(i-1)*10);
			        source.setDataTitle("百度新闻搜索——"+word+"——第"+i+"页");
					source.setDataCat(10);
					source.setDataContent(html.getTextContent());
					source.setDataIndexNo(0);
					source.setSearchDate(date);
					source.setSearchTimes(searchTimes);
					source.setUserChecked(10);
					source.setSource("BaiduWeb");
					
					srhDAO.save(source);
					for(int j=1;j<=10;j++){
						int index = j+(i-1)*10;
						Node table = doc.getElementById(""+index);
						if(table!=null){
							NodeList tbodyList = table.getChildNodes();
							int item = 0;
							Node tbody = tbodyList.item(0);
							while(!tbody.getNodeName().toUpperCase().equals("TBODY")){
								tbody = tbodyList.item(++item);
							}
							log.error("item:"+item);
							NodeList nodeList = tbody.getChildNodes();
					        Node tr = tbody.getFirstChild();
					        item =0;
					        while(!tr.getNodeName().toUpperCase().equals("TR"))
					        	tr = nodeList.item(++item);
					        log.error("item:"+item);
					        
					        Node td = tr.getFirstChild();
					        NodeList tdList = td.getChildNodes();
					        item = 0;
					        Node a = td.getFirstChild();
					        if(!a.getNodeName().toUpperCase().equals("A")){
					        	a = tdList.item(++item);
					        }
					        log.error("item:"+item);
					        String surl = a.getAttributes().getNamedItem("href").getTextContent();

					        Searchdata data = new Searchdata();
							data.setDataContent(table.getTextContent());
							data.setDataIndexNo(index);
							data.setDataPageNo(i);
							data.setDataSource(this.writeDoc(table));
							data.setDataTitle(a.getTextContent());
							log.error(a.getTextContent());
							data.setSea_data_id(source.getDataId());
							data.setSearchDate(date);
							data.setSearchTimes(searchTimes);
							data.setWordId(wordId);
					        data.setLinkUrl(surl);
					        data.setSource("BaiduWeb");
							
					        int checked = helper.checkDataCat(surl, data.getDataTitle(), wordId);
					        
							data.setUserChecked(checked);
							data.setDataCat(checked>0 ? 1 : 0);
							
						    NodeList list = td.getChildNodes().item(2).getChildNodes();
						    for(int k=0;k<list.getLength();k++){
						    	Node l = list.item(k);
						    	if(l.getNodeName().toUpperCase().equals("A")){
						    		String baidukuaizhao = l.getAttributes().getNamedItem("href").getTextContent();
						        		if(baidukuaizhao.indexOf("cache.baidu.com")>=0){
						        			log.error(baidukuaizhao);
						        			data.setBaiduCache(baidukuaizhao);
						        		}else data.setBaiduCache(surl);
						        		break;
						        	}
						   }
					        srhDAO.save(data);
						}else return ;
					}
			}
			}catch(Exception e){
			e.printStackTrace();
		}
		return ;
	}
	
	
	public void Search(String uri,Keyword keyword){
		try{
			uri = uri==null||uri.length()==0 ? "http://news.baidu.com/ns?word=%"+URLEncoder.encode(keyword.getWordName(), "GBK")
					+"%E4&ie=gb2312&bs=%CC%EC%C9%CF%C8%CB%BC%E4&sr=0&cl=2&rn=20&tn=newsA&ct=0&clk=sortbytime" : uri;
			log.error(keyword.getWordName()+uri);
			Date date = new Date();
			//String today = CalendarFunction.dateToString(date);
			 
					URL url = new URL(uri);
					
					HttpURLConnection connection = (HttpURLConnection) url.openConnection();
	                /*  
					 
		                  //URL url = new URL("http://www.ama.org/events ");
		                  connection.setFollowRedirects(true);
		                  connection.setInstanceFollowRedirects(false);
		                  connection.connect();
		             
		              while (String.valueOf(connection.getResponseCode()).startsWith("3")) {
		                 String theLocation = connection.getHeaderField("Location");
		                 connection.disconnect();
		                 url = new URL(theLocation);
		                 connection = (HttpURLConnection) url.openConnection();
		                 connection.setFollowRedirects(true);
		                 connection.setInstanceFollowRedirects(false);
		                 connection.connect();
		                 }
					
					
					
					
					*/
					//HttpURLConnection connection = (HttpURLConnection)url.openConnection();
					connection.setConnectTimeout(5000);
					InputStreamReader in = new InputStreamReader(connection.getInputStream(),"GBK");
					
					DOMParser parser = new DOMParser();
			        parser.setProperty("http://cyberneko.org/html/properties/default-encoding","GBK");
			        parser.setFeature("http://cyberneko.org/html/features/augmentations", true);
			        parser.setProperty("http://cyberneko.org/html/properties/names/elems", "lower");
			        
			        parser.parse(new InputSource(in));
			        Document doc = parser.getDocument();
			        in.close();
			        connection.disconnect();
			        System.gc();

			        NodeList as = doc.getElementsByTagName("a");
			   
			        for(int i=0;i<as.getLength();i++){
			        	Node a = as.item(i);
			        	if(a.getTextContent().equals(keyword.getWordName()+"全部新闻中重新搜索")){
			        		log.debug(keyword.getWordName()+"_全部新闻中重新搜索");
			        		this.Search("http://news.baidu.com"+a.getAttributes().getNamedItem("href").getNodeValue(),keyword);
			        	}
			        }
			        pageNum++;
			        Node html = doc.getElementsByTagName("html").item(0);
			        Searchdata source = new Searchdata();
			        source.setWordId(keyword.getWordId());
			        string = "";
			        source.setDataSource(this.writeDoc(html));
			        source.setDataPageNo(pageNum);
			        source.setLinkUrl(uri);
			        source.setDataTitle("百度新闻搜索——"+keyword.getWordName());
					source.setDataCat(10);
					source.setDataContent(html.getTextContent());
					source.setDataIndexNo(0);
					source.setSearchDate(date);
					source.setSearchTimes(1);
					source.setUserChecked(10);
					source.setSource("BaiduNews");
					
					//srhDAO.save(source);
			        NodeList tables = doc.getElementsByTagName("table");
			        log.debug(keyword.getWordName()+"长度："+tables.getLength());
			        for(int i=0;i<tables.getLength();i++){
			        	Node table = tables.item(i);
			        	string = "";
			        	//log.error(this.writeDoc(table));
			        	if(//table.getTextContent().indexOf(keyword.getWordName())>=0&&
			        			table.getAttributes().getNamedItem("cellpadding")!=null&&
			        			table.getAttributes().getNamedItem("cellspacing")!=null&&
			        			table.getAttributes().getNamedItem("cellpadding").getNodeValue().equals("2")
			        			&&table.getAttributes().getNamedItem("cellspacing").getNodeValue().equals("0")){
			        		Node tbody = table.getLastChild();
			        		Node tr = tbody.getLastChild();
			        		NodeList tds = tr.getChildNodes();
			        		
			        		string = "";
			        		
			        		for(int j=0;j<tds.getLength();j++){
			        			Node td = tds.item(j);
			        			
			        			if(td==null||td.getNodeName().toLowerCase().equals("#text"))continue;
			        			if(td.getAttributes().getNamedItem("class").getNodeValue().equals("text")){
			        				Node a = td.getFirstChild();
			        				String href = a.getAttributes().getNamedItem("href").getNodeValue();
			        				
			        				
					        		Node font = td.getChildNodes().item(2);
					        		
			        				String nobr[] = font.getChildNodes().item(1).getTextContent().split(" ");
			        				String fromWeb = nobr[0];
			        				String publishDate = nobr[1];
			        				String publishTime = nobr[2];
			        				Node font2 = td.getChildNodes().item(4);
			        				Node moreHref = td.getChildNodes().item(6);
			        				String title = a.getTextContent();
			        				String baiduCache = "";
			        				//log.error(title);
			        				if(moreHref != null && moreHref.getNodeName()!=null){
			        					//log.error(moreHref.getNodeName()+"  "+title+"    "+moreHref.getTextContent());
			        					//log.error("===================================================");
			        					if(moreHref.getAttributes().getNamedItem("href").getNodeValue().indexOf("newscache.baidu.com")>0){
			        						baiduCache = moreHref.getAttributes().getNamedItem("href").getNodeValue();
			        					}else{
			        						this.subSearch("http://news.baidu.com"+moreHref.getAttributes().getNamedItem("href").getNodeValue(), 
			        							keyword.getWordId(), keyword.getWordName());
			        						continue;
			        					}
			        					
			        					//log.error(moreHref.getAttributes().getNamedItem("href").getNodeValue());
			        					//log.error("=====================================================");
			        					
			        				}
			        				if(helper.checkBaiduNewsSaved(href,keyword.getWordId())){
			        					try{
			        					//log.error(title+"    已保存过");
			        					}catch(Exception e){
			        						string = "";
			        						//log.error(this.writeDoc(table));
			        						e.printStackTrace();
			        					}
			        					if(!continueNews){
			        						continueNews = true;
			        						sameNews = 0;
			        					}
			        					//sameNews ++;
			        					continue;
			        				}
			        				continueNews = false;
			        				String detail = font2.getTextContent();
			        				
			        				//log.error(title);
			        				Searchdata data = new Searchdata();
									data.setDataContent(detail);
									data.setDataIndexNo(0);
									data.setDataPageNo(0);
									data.setBaiduCache(baiduCache);
									string = "";
									String tableSource = this.writeDoc(table);
									//log.error("tablesource:"+tableSource.length());
									data.setDataSource(tableSource);
									data.setDataTitle(title);
									//log.error("标题："+title+"来源："+fromWeb+"url:"+href);
									//data.setSea_data_id(source.getDataId());
									data.setSearchDate(CalendarFunction.StringToDate2(publishDate+" "+publishTime+":00"));
									data.setSearchTimes(1);
									data.setWordId(keyword.getWordId());
							        data.setLinkUrl(href);
							        data.setSource("BaiduNews"+fromWeb);
									
							        int checked = helper.checkDataCat(href, data.getDataTitle(), keyword.getWordId());
							        
									data.setUserChecked(0);
									data.setDataCat(checked>0 ? 0 : 0);
									srhDAO.save(data);
			        			}
			        		}
			        	}
			        }
			        if(pageNum >= keyword.getCountPageNo()) return;
			        for(int i=0;i<as.getLength();i++){
			        	Node a = as.item(i);
			        	if(a.getTextContent().equals("下一页")){
			        		if(continueNews&&sameNews>=30)
			        			break;
			        		log.debug(keyword.getWordName()+"_下一页");
			        		this.Search("http://news.baidu.com"+a.getAttributes().getNamedItem("href").getNodeValue(),keyword);
			        	}
			        }
			}catch(Exception e){
			e.printStackTrace();
		}
	        
		return ;
	
		
		
	}
	
	public void SearchByMonth(String uri,Keyword keyword){
		try{
			uri = uri==null||uri.length()==0 ? "http://news.baidu.com/ns?word=%"+URLEncoder.encode(keyword.getWordName(), "GBK")
					+"%E4&ie=gb2312&bs=%CC%EC%C9%CF%C8%CB%BC%E4&sr=0&cl=2&rn=20&tn=newsA&ct=0&clk=sortbytime" : uri;
			log.error(keyword.getWordName()+uri);
			Date date = new Date();
			//String today = CalendarFunction.dateToString(date);
					
					URL url = new URL(uri);
					
					HttpURLConnection connection = (HttpURLConnection)url.openConnection();
					connection.setConnectTimeout(5000);
					InputStreamReader in = new InputStreamReader(connection.getInputStream(),"GB18030");
					
					DOMParser parser = new DOMParser();
			        parser.setProperty("http://cyberneko.org/html/properties/default-encoding","GB18030");
			        parser.setFeature("http://cyberneko.org/html/features/augmentations", true);
			        parser.setProperty("http://cyberneko.org/html/properties/names/elems", "lower");
			        
			        parser.parse(new InputSource(in));
			        Document doc = parser.getDocument();
			        in.close();
			        connection.disconnect();

			        NodeList as = doc.getElementsByTagName("a");
			   
			        for(int i=0;i<as.getLength();i++){
			        	Node a = as.item(i);
			        	if(a.getTextContent().equals(keyword.getWordName()+"全部新闻中重新搜索")){
			        		log.debug(keyword.getWordName()+"_全部新闻中重新搜索");
			        		this.Search("http://news.baidu.com"+a.getAttributes().getNamedItem("href").getNodeValue(),keyword);
			        	}
			        }
			        pageNum++;
			        Node html = doc.getElementsByTagName("html").item(0);
			        Searchdata source = new Searchdata();
			        source.setWordId(keyword.getWordId());
			        string = "";
			        source.setDataSource(this.writeDoc(html));
			        source.setDataPageNo(pageNum);
			        source.setLinkUrl(uri);
			        source.setDataTitle("百度新闻搜索——"+keyword.getWordName());
					source.setDataCat(10);
					source.setDataContent(html.getTextContent());
					source.setDataIndexNo(0);
					source.setSearchDate(date);
					source.setSearchTimes(1);
					source.setUserChecked(10);
					source.setSource("BaiduNews");
					
					//srhDAO.save(source);
			        NodeList tables = doc.getElementsByTagName("table");
			        log.debug(keyword.getWordName()+"长度："+tables.getLength());
			        for(int i=0;i<tables.getLength();i++){
			        	Node table = tables.item(i);
			        	string = "";
			        	//log.error(this.writeDoc(table));
			        	if(//table.getTextContent().indexOf(keyword.getWordName())>=0&&
			        			table.getAttributes().getNamedItem("cellpadding")!=null&&
			        			table.getAttributes().getNamedItem("cellspacing")!=null&&
			        			table.getAttributes().getNamedItem("cellpadding").getNodeValue().equals("2")
			        			&&table.getAttributes().getNamedItem("cellspacing").getNodeValue().equals("0")){
			        		Node tbody = table.getLastChild();
			        		Node tr = tbody.getLastChild();
			        		NodeList tds = tr.getChildNodes();
			        		
			        		string = "";
			        		
			        		for(int j=0;j<tds.getLength();j++){
			        			Node td = tds.item(j);
			        			
			        			if(td==null||td.getNodeName().toLowerCase().equals("#text"))continue;
			        			if(td.getAttributes().getNamedItem("class").getNodeValue().equals("text")){
			        				Node a = td.getFirstChild();
			        				String href = a.getAttributes().getNamedItem("href").getNodeValue();
			        				
			        				
					        		Node font = td.getChildNodes().item(2);
					        		
			        				String nobr[] = font.getChildNodes().item(1).getTextContent().split(" ");
			        				String fromWeb = nobr[0];
			        				String publishDate = nobr[1];
			        				String publishTime = nobr[2];
			        				Node font2 = td.getChildNodes().item(4);
			        				Node moreHref = td.getChildNodes().item(6);
			        				String title = a.getTextContent();
			        				//log.error(title);
			        				if(moreHref != null && moreHref.getNodeName()!=null){
			        					//log.error(moreHref.getNodeName()+"  "+title+"    "+moreHref.getTextContent());
			        					//log.error("===================================================");
			        					this.subSearch("http://news.baidu.com"+moreHref.getAttributes().getNamedItem("href").getNodeValue(), 
			        							keyword.getWordId(), keyword.getWordName());
			        					continue;
			        					//log.error(moreHref.getAttributes().getNamedItem("href").getNodeValue());
			        					//log.error("=====================================================");
			        					
			        				}
			        				if(helper.checkBaiduNewsSaved(href,keyword.getWordId())){
			        					try{
			        					//log.error(title+"    已保存过");
			        					}catch(Exception e){
			        						string = "";
			        						//log.error(this.writeDoc(table));
			        						e.printStackTrace();
			        					}
			        					if(!continueNews){
			        						continueNews = true;
			        						sameNews = 0;
			        					}
			        					//sameNews ++;
			        					continue;
			        				}
			        				continueNews = false;
			        				String detail = font2.getTextContent();
			        				
			        				//log.error(title);
			        				Searchdata data = new Searchdata();
									data.setDataContent(detail);
									data.setDataIndexNo(0);
									data.setDataPageNo(0);
									string = "";
									String tableSource = this.writeDoc(table);
									//log.error("tablesource:"+tableSource.length());
									data.setDataSource(tableSource);
									data.setDataTitle(title);
									//log.error("标题："+title+"来源："+fromWeb+"url:"+href);
									//data.setSea_data_id(source.getDataId());
									data.setSearchDate(CalendarFunction.StringToDate2(publishDate+" "+publishTime+":00"));
									data.setSearchTimes(1);
									data.setWordId(keyword.getWordId());
							        data.setLinkUrl(href);
							        data.setSource("BaiduNews"+fromWeb);
									
							        int checked = helper.checkDataCat(href, data.getDataTitle(), keyword.getWordId());
							        
									data.setUserChecked(0);
									data.setDataCat(checked>0 ? 0 : 0);
									srhDAO.save(data);
			        			}
			        		}
			        	}
			        }
			        if(pageNum >= keyword.getCountPageNo()) return;
			        for(int i=0;i<as.getLength();i++){
			        	Node a = as.item(i);
			        	if(a.getTextContent().equals("下一页")){
			        		if(continueNews&&sameNews>=30)
			        			break;
			        		log.debug(keyword.getWordName()+"_下一页");
			        		this.Search("http://news.baidu.com"+a.getAttributes().getNamedItem("href").getNodeValue(),keyword);
			        	}
			        }
			}catch(Exception e){
			e.printStackTrace();
		}
		return ;
	
		
		
	}
	
	public void subSearch(String uri,int wordId,String keyword){
		try{
			//log.error(uri);
			Date date = new Date();
			//String today = CalendarFunction.dateToString(date);
					
					URL url = new URL(uri);
					
					HttpURLConnection connection = (HttpURLConnection)url.openConnection();
					connection.setConnectTimeout(5000);
					InputStreamReader in = new InputStreamReader(connection.getInputStream(),"GB18030");
					
					DOMParser parser = new DOMParser();
			        parser.setProperty("http://cyberneko.org/html/properties/default-encoding","GB18030");
			        parser.setFeature("http://cyberneko.org/html/features/augmentations", true);
			        parser.setProperty("http://cyberneko.org/html/properties/names/elems", "lower");
			        
			        parser.parse(new InputSource(in));
			        Document doc = parser.getDocument();
			        in.close();
			        connection.disconnect();
			        
			        NodeList as = doc.getElementsByTagName("a");
			   
			        for(int i=0;i<as.getLength();i++){
			        	Node a = as.item(i);
			        	if(a.getTextContent().equals("全部新闻中重新搜索")){
			        		//log.debug("全部新闻中重新搜索");
			        		this.subSearch("http://news.baidu.com"+a.getAttributes().getNamedItem("href").getNodeValue(),wordId,keyword);
			        	}
			        }
			        Node html = doc.getElementsByTagName("html").item(0);
			        Searchdata source = new Searchdata();
			        source.setWordId(wordId);
			        string = "";
			        source.setDataSource(this.writeDoc(html));
			        //source.setDataPageNo(i);
			        source.setLinkUrl(uri);
			        source.setDataTitle("百度新闻搜索——"+keyword);
					source.setDataCat(10);
					source.setDataContent(html.getTextContent());
					source.setDataIndexNo(0);
					source.setSearchDate(date);
					source.setSearchTimes(1);
					source.setUserChecked(10);
					source.setSource("BaiduNews");
					
					//srhDAO.save(source);
			        NodeList tables = doc.getElementsByTagName("table");
			        //log.debug("长度："+tables.getLength());
			        for(int i=0;i<tables.getLength();i++){
			        	Node table = tables.item(i);
			        	if(//table.getTextContent().indexOf(keyword.getWordName())>=0&&
			        			table.getAttributes().getNamedItem("cellpadding")!=null&&
			        			table.getAttributes().getNamedItem("cellspacing")!=null&&
			        			table.getAttributes().getNamedItem("cellpadding").getNodeValue().equals("2")
			        			&&table.getAttributes().getNamedItem("cellspacing").getNodeValue().equals("0"))
			        		{
			        		
			        		Node tbody = table.getLastChild();
			        		
			        		Node tr = tbody.getLastChild();
			        		
			        		NodeList tds = tr.getChildNodes();
			        		
			        		for(int j=0;j<tds.getLength();j++){
			        			Node td = tds.item(j);
			        			//log.debug("td:"+td.getNodeName());
			        			if(td==null||td.getNodeName().toLowerCase().equals("#text"))continue;
			        			
			        			if(td.getAttributes().getNamedItem("class").getNodeValue().equals("text")){
			        				
			        				Node a = td.getFirstChild();
					        		
			        				String href = a.getAttributes().getNamedItem("href").getNodeValue();
			        				
			        				if(helper.checkBaiduNewsSaved(href,wordId))continue;
			        				
					        		Node font = td.getChildNodes().item(2);
			        				String nobr[] = font.getChildNodes().item(1).getTextContent().split(" ");
			        				String fromWeb = nobr[0];
			        				String publishDate = nobr[1];
			        				String publishTime = nobr[2];
			        				String baiduCache = "";
			        				if(td.getChildNodes().item(6)!=null && td.getChildNodes().item(6).getNodeName()!=null){
			        					baiduCache = td.getChildNodes().item(6).getNodeValue();
			        				}
			        				//string = "";
			        				//log.error(this.writeDoc(td));
			        				Node font2 = td.getChildNodes().item(4);
			        				String detail = font2.getTextContent();
			        				String title = a.getTextContent();
			        				//log.error(detail);

					        		Searchdata data = new Searchdata();
									data.setDataContent(detail);
									data.setDataIndexNo(0);
									data.setDataPageNo(0);
									string = "";
									data.setDataSource(this.writeDoc(table));
									data.setDataTitle(title);
									//data.setSea_data_id(source.getDataId());
									data.setSearchDate(CalendarFunction.StringToDate2(publishDate+" "+publishTime+":00"));
									data.setSearchTimes(1);
									data.setWordId(wordId);
							        data.setLinkUrl(href);
							        data.setBaiduCache(baiduCache);
							        data.setSource("BaiduNews"+fromWeb);
									
							        int checked = helper.checkDataCat(href, data.getDataTitle(), wordId);
							        
									data.setUserChecked(0);
									data.setDataCat(checked>0 ? 0 : 0);
									srhDAO.save(data);
			        			}
			        		}
			        	}
			        }
			        
			        for(int i=0;i<as.getLength();i++){
			        	Node a = as.item(i);
			        	if(a.getTextContent().equals("下一页")){
			        		//log.debug("下一页");
			        		this.subSearch("http://news.baidu.com"+a.getAttributes().getNamedItem("href").getNodeValue(),wordId,keyword);
			        	}
			        }
			}catch(Exception e){
				log.error("关键词："+keyword+"错误链接地址："+uri);
			e.printStackTrace();
		}
		return ;
	
		
		
	}
	private String writeDoc(Node node) {
		  short type = node.getNodeType();
		  switch (type) {

		  case Node.ELEMENT_NODE: {
		   String name = "<" + node.getNodeName();
		   NamedNodeMap attrs = node.getAttributes();
		   if (attrs != null) {
		    int length = attrs.getLength();
		    for (int i = 0; i < length; i++) {
		     Node attr = attrs.item(i);
		     name += " " + attr.getNodeName();
		     name += "=\"" + attr.getNodeValue() + "\"";
		    }
		   }
		   name += ">";
		   string += name;
		   //System.out.println(name);

		   NodeList children = node.getChildNodes();
		   if (children != null) {
		    int length = children.getLength();
		    for (int i = 0; i < length; i++)
		     writeDoc(children.item(i));
		   }
		   
		   string += ("</" + node.getNodeName() + ">");
		   break;
		  }
		  case Node.TEXT_NODE: {
		   string += (node.getNodeValue());
		   break;
		  }
		  }
		  return string;
		 }
	
	public void init(){
		User user = new User();
		user.setRegDate(new Date());
		user.setUserName("tuxiliang");
		user.setUserPwd("123456");
		user.setEndDate(new Date());
		//userDAO.save(user);
		
		Company comp = new Company();
		comp.setCompEnddate(CalendarFunction.StringToDate("2010-06-09"));
		comp.setCompName("星美");
		comp.setJoinDate(new Date());
		comp.setUserId(user.getUserId());
		companyDAO.save(comp);
		
		Project proj = new Project();
		proj.setCompId(comp.getCompId());
		proj.setProjAddDate(new Date());
		proj.setProjDesc("项目描述");
		proj.setProjEnddate(CalendarFunction.StringToDate("2010-06-09"));
		proj.setProjName("公司关键词优化");
		proj.setUserId(user.getUserId());
		projectDAO.save(proj);
		Project proj2 = new Project();
		proj2.setCompId(comp.getCompId());
		proj2.setProjAddDate(new Date());
		proj2.setProjDesc("项目描述");
		proj2.setProjEnddate(CalendarFunction.StringToDate("2010-06-09"));
		proj2.setProjName("个人关键词优化");
		proj2.setUserId(user.getUserId());
		projectDAO.save(proj2);
		
		
		Keyword keyword = new Keyword();
		keyword.setAddWordDate(new Date());
		keyword.setCountPageNo(10);
		keyword.setProjId(proj.getProjId());
		keyword.setUserId(user.getUserId());
		keyword.setWordName("星美");
		keyword.setEndWordDate(new Date());
		keywordDAO.save(keyword);
		
		Keyword keyword3 = new Keyword();
		keyword3.setAddWordDate(new Date());
		keyword3.setCountPageNo(10);
		keyword3.setProjId(proj2.getProjId());
		keyword3.setUserId(user.getUserId());
		keyword3.setWordName("覃辉");
		keywordDAO.save(keyword3);
		
		
	}
	
	public static BaiduNewsSpider getFromApplicationContext(ApplicationContext ctx) {
		return (BaiduNewsSpider) ctx.getBean("BaiduNewsSpider");
	}
	public static void main(String[] args) throws UnsupportedEncodingException {
		ApplicationContext appContext = new FileSystemXmlApplicationContext("/src/applicationContext.xml");
		BaiduNewsSpider spider =  BaiduNewsSpider.getFromApplicationContext(appContext);
		/*Keyword keyword = new Keyword();
		keyword.setWordId(20);
		keyword.setCountPageNo(30);
		keyword.setWordName("特发信息");
		//spider.subSearch("http://news.baidu.com/ns?word=%CC%EC%C9%CF%C8%CB%BC%E4+cont:1036135923&cl=1&tn=news&rn=30", 1,"天上人间");
		
		spider.Search(null,keyword);
		*/
		/*Keyword keyword2 = new Keyword();
		keyword2.setWordId(21);
		keyword2.setCountPageNo(30);
		keyword2.setWordName("中天科技");
		//spider.subSearch("http://news.baidu.com/ns?word=%CC%EC%C9%CF%C8%CB%BC%E4+cont:1036135923&cl=1&tn=news&rn=30", 1,"天上人间");
		
		spider.Search(null,keyword2);*/
		
		/*Keyword keyword3 = new Keyword();
		keyword3.setWordId(22);
		keyword3.setCountPageNo(30);
		keyword3.setWordName("永鼎股份");
		//spider.subSearch("http://news.baidu.com/ns?word=%CC%EC%C9%CF%C8%CB%BC%E4+cont:1036135923&cl=1&tn=news&rn=30", 1,"天上人间");
		
		spider.Search(null,keyword3);*/
		
		/*Keyword keyword4 = new Keyword();
		keyword4.setWordId(23);
		keyword4.setCountPageNo(30);
		keyword4.setWordName("亨通光电");
		//spider.subSearch("http://news.baidu.com/ns?word=%CC%EC%C9%CF%C8%CB%BC%E4+cont:1036135923&cl=1&tn=news&rn=30", 1,"天上人间");
		
		spider.Search(null,keyword4);*/
		
		/*Keyword keyword5 = new Keyword();
		keyword5.setWordId(4);
		keyword5.setCountPageNo(10);
		keyword5.setWordName("法尔胜");
		//spider.subSearch("http://news.baidu.com/ns?word=%CC%EC%C9%CF%C8%CB%BC%E4+cont:1036135923&cl=1&tn=news&rn=30", 1,"天上人间");
		
		spider.Search(null,keyword5);*/
		
		/*Keyword keyword6 = new Keyword();
		keyword6.setWordId(25);
		keyword6.setCountPageNo(1);
		keyword6.setWordName("汇源光通信");
		//spider.subSearch("http://news.baidu.com/ns?word=%CC%EC%C9%CF%C8%CB%BC%E4+cont:1036135923&cl=1&tn=news&rn=30", 1,"天上人间");
		
		spider.Search(null,keyword6);
		*/
		
		Keyword keyword6 = new Keyword();
		keyword6.setWordId(3);
		keyword6.setCountPageNo(15);
		keyword6.setWordName("成清波");
		//spider.subSearch("http://news.baidu.com/ns?word=%CC%EC%C9%CF%C8%CB%BC%E4+cont:1036135923&cl=1&tn=news&rn=30", 1,"天上人间");
		
		spider.Search(null,keyword6);
		//Keyword keyword = spider.keywordDAO.findById(4);
		//spider.search(keyword);
		//spider.runSpiderOneByOne();
		//spider.runSpiderWithThread();
		//spider.init();
	}
	
}
